In [166]:
%matplotlib inline
import numpy as np
import pandas as pd
import pickle
import matplotlib.pyplot as plt
import seaborn as sns
from pandas import DataFrame
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import Imputer
from sklearn.pipeline import Pipeline
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.grid_search import GridSearchCV
from sklearn.cross_validation import StratifiedKFold
from sklearn.base import TransformerMixin
from sklearn.feature_selection import VarianceThreshold
from sklearn.feature_selection import SelectPercentile
from sklearn.feature_selection import SelectKBest
from sklearn.cross_validation import cross_val_score
from sklearn.metrics import f1_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.feature_selection import f_classif
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.decomposition import PCA
from sklearn.cross_validation import StratifiedShuffleSplit
from sklearn.svm import LinearSVC
from sklearn.neighbors import KNeighborsRegressor
from sklearn.pipeline import FeatureUnion
In [167]:
data_dict = pickle.load(open("../ud120-projects/final_project/final_project_dataset.pkl", "r") )
Since the dataset for this project is so small, a hold-out set will not be used, and only k-fold testing and training splits will be used to measure accuracy.
This is because even with a stratified hold-out set of 20%, with only 146 data points, lots of missing data and and 18 poi's, there would be only 3 or so points to do a final test on. This does not give much confidence in the precision of the performance metrics on such a small hold-out set, while also negatively impacting the ability to create the model.
"when the number of samples is not large, a strong case can be made that a test set should be avoided because every sample may be needed for model building. (...) Additionally, the size of the test set may not have sufficient power or precision to make reasonable judgements. "
[1] Kuhn M., Kjell J.(2013). Applied Predictive Modeling. Springer. pp.67
Hawkins et al. (2003) concisely summarize this point:“holdout samples of tolerable size [. . . ] do not match the cross-validation itself for reliability in assessing model fit and are hard to motivate.”
[2] Hawkins D, Basak S, Mills D (2003). “Assessing Model Fit by Cross– Validation.” Journal of Chemical Information and Computer Sciences, 43(2), 579–586
This will be addressed with K-fold cross-validation resampling techniques.
In [168]:
df = pd.DataFrame.from_dict(data_dict, orient='index')
'NaN' was imported as a string instead of a a missing value. We will convert these to NaN type and look how many missing values our data has.
In [169]:
# Replace 'NaN' strings with actual np.nan values
df = df.replace('NaN', np.nan)
# Replace email strings with True/False boolean as to whether an email was present or not
df['email_address'] = df['email_address'].fillna(0).apply(lambda x: x != 0, 1)
In [170]:
# Replace with index watcher
# A quick look at the original finanical spreadsheet shows TOTAL at the bottom
# totaling all entries for everyone. This is obviously an outlier with no
# meaningful information and can be removed.
# df[df['salary'] > 1000000]
# df[df.index == 'TOTAL']
df = df.drop('TOTAL', axis=0)
In [170]:
By default, the GridSearchCV uses a 3-fold cross-validation. However, if it detects that a classifier is passed, rather than a regressor, it uses a stratified 3-fold.
http://scikit-learn.org/stable/tutorial/statistical_inference/model_selection.html
In [171]:
# low_var_remover = VarianceThreshold(threshold=.5)
In [172]:
# ************************
# Encode as 0 instead.
# Remove columns with more than 50% NA's
# df_50 = df.dropna(axis=1, thresh=len(df)/2)
# ************************
# Since email_address and poi are True/False, every record should have at least 2 non-NA.
# We'll next remove any rows that don't have at least 2 non-NA values besides these.
# The criteria is: No more than 11 NA's per row.
# df_50 = df_50.dropna(axis=0, thresh=5)
# 128 records remain.
# df_50.info()
When looking at the source of the data, the NA entries in the financial data seem values that are reported as zero since all payments/stock values add up to the total payments/stocks values. These NA values should then be set to 0 to add up to the totals reported by the accounting spreadsheet.
The missing values for NA's for email statistics may be a little more subjective.
Some email statistics are features created with prior knowledge of the entire dataset (i.e. emails to/from poi's). This may be data snooping, since if new data/pois were somehow introduced, it would not be possible to generate these features without prior knowledge of which new data were the poi's.
NA's here imply that the person did not have an email account with Enron, or were not involved in emailing by some other way.
This means all email data features ar NA if even one column had missing email data for that person. It is hard to judge any distribution that they could have if they were given an email account since they have no ties to the financial data to infer distributions.
We have no real way to infer a person having sent/recieved 10 emails or 10,000 from completely unrelated financial data from a different dataset with many different people.
For this reason, these NA will also be encoded as 0.
In [173]:
df = df.apply(lambda x: x.fillna(0), axis=0)
In [173]:
In [174]:
import seaborn as sns
sns.set(style='darkgrid')
f, ax = plt.subplots(figsize=(14, 14))
cmap = sns.diverging_palette(10, 220, as_cmap=True)
sns.corrplot(df.corr(), annot=True, sig_stars=False,
diag_names=False, cmap=cmap, ax=ax)
f.tight_layout()
In [175]:
# Pick a column which we are predicting.
# Find other variables correlated to used KMeansNeighborsRegression to predict/impute
# the missing values.
# df_50.corr().ix[: ,'salary']
In [176]:
# cols1 = ['salary', 'other', 'total_stock_value', 'exercised_stock_options',
# 'total_payments', 'restricted_stock']
# Bonus and salary values don't seem to be missing at random. Anytime there is a null value
# for salary, there is also one for bonus. So bonus can't be used to predict salary on
# the first pass. Predicted salary values will be used to predict bonus values though
# on a second pass.
# cols2= ['salary', 'other', 'total_stock_value', 'exercised_stock_options',
# 'total_payments', 'restricted_stock', 'bonus']
# cols3 = ['to_messages', 'from_this_person_to_poi', 'from_messages',
# 'shared_receipt_with_poi', 'from_poi_to_this_person']
In [177]:
def kcluster_null(df=None, cols=None, process_all=True):
'''
Input: Takes pandas dataframe with values to impute, and a list of columns to impute
and use for imputing.
Returns: Pandas dataframe with null values imputed for list of columns passed in.
# Ideally columns should be somewhat correlated since they will be used in KNN to
# predict each other, one column at a time.
'''
# Create a KNN regression estimator for
income_imputer = KNeighborsRegressor(n_neighbors=1)
# Loops through the columns passed in to impute each one sequentially.
if not process_all:
to_pred = cols[0]
predictor_cols = cols[1:]
for each in cols:
# Create a temp list that does not include the column being predicted.
temp_cols = [col for col in cols if col != each]
# Create a dataframe that contains no missing values in the columns being predicted.
# This will be used to train the KNN estimator.
df_col = df[df[each].isnull()==False]
# Create a dataframe with all of the nulls in the column being predicted.
df_null_col = df[df[each].isnull()==True]
# Create a temp dataframe filling in the medians for each column being used to
# predict that is missing values.
# This step is needed since we have so many missing values distributed through
# all of the columns.
temp_df_medians = df_col[temp_cols].apply(lambda x: x.fillna(x.median()), axis=0)
# Fit our KNN imputer to this dataframe now that we have values for every column.
income_imputer.fit(temp_df_medians, df_col[each])
# Fill the df (that has null values being predicted) with medians in the other
# columns not being predicted.
# ** This currently uses its own medians and should ideally use the predictor df's
# ** median values to fill in NA's of columns being used to predict.
temp_null_medians = df_null_col[temp_cols].apply(lambda x: x.fillna(x.median()), axis=0)
# Predict the null values for the current 'each' variable.
new_values = income_imputer.predict(temp_null_medians[temp_cols])
# Replace the null values of the original null dataframe with the predicted values.
df_null_col[each] = new_values
# Append the new predicted nulls dataframe to the dataframe which containined
# no null values.
# Overwrite the original df with this one containing predicted columns.
# Index order will not be preserved since it is rearranging each time by
# null values.
df = df_col.append(df_null_col)
# Returned final dataframe sorted by the index names.
return df.sort_index(axis=0)
In [177]:
In [178]:
df.irow(127)
Out[178]:
In [179]:
#cols = [x for x in df.columns]
#for each in cols:
# g = sns.FacetGrid(df, col='poi', margin_titles=True, size=6)
# g.map(plt.hist, each, color='steelblue')
In [180]:
from pandas.tools.plotting import scatter_matrix
In [181]:
list(df.columns)
Out[181]:
In [182]:
financial_cols = np.array(['salary', 'deferral_payments', 'total_payments', 'exercised_stock_options',
'bonus', 'restricted_stock', 'restricted_stock_deferred', 'total_stock_value',
'expenses', 'loan_advances', 'other', 'director_fees', 'deferred_income',
'long_term_incentive'])
email_cols = np.array(['from_messages', 'to_messages', 'shared_receipt_with_poi',
'from_this_person_to_poi', 'from_poi_to_this_person', 'email_address'])
In [183]:
from sklearn.ensemble import RandomForestClassifier
In [184]:
clf = RandomForestClassifier(n_estimators=1000)
clf.fit(df[financial_cols], df['poi'])
Out[184]:
In [185]:
importances = clf.feature_importances_
sorted_idx = np.argsort(importances)
In [186]:
padding = np.arange(len(financial_cols)) + 0.5
plt.figure(figsize=(14, 12))
plt.barh(padding, importances[sorted_idx], align='center')
plt.yticks(padding, financial_cols[sorted_idx])
plt.xlabel("Relative Importance")
plt.title("Variable Importance")
plt.show()
In [187]:
clf = RandomForestClassifier(n_estimators=1000)
clf.fit(df[email_cols], df['poi'])
importances = clf.feature_importances_
sorted_idx = np.argsort(importances)
padding = np.arange(len(email_cols)) + 0.5
plt.figure(figsize=(14, 12))
plt.barh(padding, importances[sorted_idx], align='center')
plt.yticks(padding, email_cols[sorted_idx])
plt.xlabel("Relative Importance")
plt.title("Variable Importance")
plt.show()
In [188]:
all_cols = np.concatenate([email_cols, financial_cols])
clf = RandomForestClassifier(n_estimators=1000)
clf.fit(df[all_cols], df['poi'])
importances = clf.feature_importances_
sorted_idx = np.argsort(importances)
padding = np.arange(len(all_cols)) + 0.5
plt.figure(figsize=(14, 12))
plt.barh(padding, importances[sorted_idx], align='center')
plt.yticks(padding, all_cols[sorted_idx])
plt.xlabel("Relative Importance")
plt.title("Variable Importance")
plt.show()
In [189]:
df['ex_stock_bins'] = pd.cut(df.exercised_stock_options, bins=15, labels=False)
pd.value_counts(df.ex_stock_bins)
Out[189]:
In [190]:
df.exercised_stock_options.plot()
Out[190]:
In [191]:
def capValues(x, cap):
return (cap if x > cap else x)
In [192]:
df.exercised_stock_options = df.exercised_stock_options.apply(lambda x: capValues(x, 5000000))
In [193]:
df['ex_stock_bins'] = pd.cut(df.exercised_stock_options, bins=15, labels=False)
pd.value_counts(df.ex_stock_bins)
Out[193]:
In [194]:
df[['ex_stock_bins', 'poi']].groupby('ex_stock_bins').mean().plot()
Out[194]:
In [195]:
df.columns
Out[195]:
In [196]:
df[['bonus', 'poi']].groupby('bonus').mean().plot()
Out[196]:
In [197]:
df.shared_receipt_with_poi.plot()
Out[197]:
In [198]:
max(df.shared_receipt_with_poi)
Out[198]:
In [199]:
# Create bins for shared receipt with poi
my_bins = [min(df.shared_receipt_with_poi)] + [250] + range(500, 5000, 500) + [max(df.shared_receipt_with_poi)]
df['shared_poi_bins'] = pd.cut(df.shared_receipt_with_poi, bins=my_bins, labels=False, include_lowest=True)
pd.value_counts(df['shared_poi_bins'])
Out[199]:
In [199]:
In [200]:
df[['shared_poi_bins', 'poi']].groupby('shared_poi_bins').mean().plot()
Out[200]:
In [201]:
df.total_stock_value
Out[201]:
In [201]:
In [202]:
from sklearn.preprocessing import StandardScaler
df['total_stock_scaled'] = StandardScaler().fit_transform(df[['total_stock_value']])
df['bonus_scaled'] = StandardScaler().fit_transform(df[['bonus']])
print df.total_stock_scaled.describe() plt.hist(df.total_stock_scaled)
In [203]:
def dont_neg_log(x):
if x >=0:
return np.log1p(x)
else:
return 0
df['stock_log'] = df['total_stock_value'].apply(lambda x: dont_neg_log(x))
In [204]:
financial_cols = np.array(['salary', 'deferral_payments', 'total_payments', 'exercised_stock_options',
'bonus', 'restricted_stock', 'restricted_stock_deferred', 'total_stock_value',
'expenses', 'loan_advances', 'other', 'director_fees', 'deferred_income',
'long_term_incentive'])
email_cols = np.array(['from_messages', 'to_messages', 'shared_receipt_with_poi',
'from_this_person_to_poi', 'from_poi_to_this_person', 'email_address'])
In [205]:
payment_comp = ['salary', 'deferral_payments','bonus', 'expenses', 'loan_advances',
'other', 'director_fees', 'deferred_income', 'long_term_incentive']
payment_total = ['total_payments']
stock_comp = ['exercised_stock_options', 'restricted_stock','restricted_stock_deferred',]
stock_total = ['total_stock_value']
all_comp = payment_comp + stock_comp
email_comp = ['shared_receipt_with_poi', 'from_this_person_to_poi', 'from_poi_to_this_person' ]
email_totals = ['from_messages', 'to_messages'] # interaction_w_poi = total(from/to/shared poi)
In [205]:
In [206]:
df['total_compensation'] = df['total_payments'] + df['total_stock_value']
for each in payment_comp:
df['{0}_{1}_ratio'.format(each, 'total_pay')] = df[each]/df['total_payments']
for each in stock_comp:
df['{0}_{1}_ratio'.format(each, 'total_stock')] = df[each]/df['total_stock_value']
for each in all_comp:
df['{0}_{1}_ratio'.format(each, 'total_compensation')] = df[each]/df['total_compensation']
df['total_poi_interaction'] = df['shared_receipt_with_poi'] + df['from_this_person_to_poi'] + \
df['from_poi_to_this_person']
for each in email_comp:
df['{0}_{1}_ratio'.format(each, 'total_poi_int')] = df[each]/df['total_poi_interaction']
df['total_active_poi_interaction'] = df['from_this_person_to_poi'] + df['from_poi_to_this_person']
df['to_poi_total_active_poi_ratio'] = df['from_this_person_to_poi']/df['total_active_poi_interaction']
df['from_poi_total_active_poi_ratio'] = df['from_poi_to_this_person']/df['total_active_poi_interaction']
df['to_messages_to_poi_ratio'] = df['from_this_person_to_poi']/ df['to_messages']
df['from_messages_from_poi_ratio'] = df['from_poi_to_this_person']/df['from_messages']
df['shared_poi_from_messages_ratio'] = df['shared_receipt_with_poi']/df['from_messages']
A good portion of people were paid either only in stock or payments. Another good portion also didn't have email statistics available.
These ratios will need to be set to zero manually due to division by 0 - NaN.
In [207]:
df = df.apply(lambda x: x.fillna(0), axis=0)
In [209]:
df[['poi', 'from_messages', 'to_messages', 'shared_receipt_with_poi','total_active_poi_interaction']]
Out[209]:
In [210]:
df[df['poi']==True]
Out[210]:
director_fees_total_pay_ratio, deferred_income_total_pay_ratio, exercised_stock_options_total_stock_ratio, exercised_stock_options_total_stock_ratio, restricted_stock_deferred_total_stock_ratio, restricted_stock_total_stock_ratio, director_fees_total_compensation_ratio, deferred_income_total_compensation_ratio, restricted_stock_total_compensation_ratio, restricted_stock_deferred_total_compensation_ratio
In [238]:
# Column slicing by number
df.ix[:,5:10].describe()
Out[238]:
In [212]:
#all_cols2 = np.concatenate([all_cols, np.array(['shared_poi_bins', 'ex_stock_bins',
# 'total_stock_scaled', 'bonus_scaled',
# 'stock_log'])])
features = np.array(df.drop('poi', axis=1).columns)
clf = ExtraTreesClassifier(n_estimators=2000)
clf.fit(df[features], df['poi'])
importances = clf.feature_importances_
sorted_idx = np.argsort(importances)
padding = np.arange(len(features)) + 0.5
plt.figure(figsize=(14,12))
plt.barh(padding, importances[sorted_idx], align='center')
plt.yticks(padding, features[sorted_idx])
plt.xlabel('Relative Importance')
plt.title('Variable Importance')
plt.show()
In [91]:
confusion_matrix(df['poi'], clf.predict(df))
Out[91]:
In [43]:
X_df = df[all_cols2]
y_df = df['poi']
In [44]:
FINANCIAL_FIELDS = ['salary', 'deferral_payments', 'total_payments', 'exercised_stock_options',
'bonus', 'restricted_stock', 'restricted_stock_deferred', 'total_stock_value',
'expenses', 'loan_advances', 'other', 'director_fees', 'deferred_income',
'long_term_incentive', 'ex_stock_bins', 'stock_log']
EMAIL_FIELDS = ['from_messages', 'to_messages', 'shared_receipt_with_poi',
'from_this_person_to_poi', 'from_poi_to_this_person', 'email_address',
'shared_poi_bins']
In [56]:
class ColumnExtractor(TransformerMixin):
'''
Columns extractor transformer for sklearn pipelines.
Inherits fit_transform() from TransformerMixin, but this is explicitly
defined here for clarity.
Methods to extract pandas dataframe columns are defined for this class.
'''
def __init__(self, columns=[]):
self.columns = columns
def fit_transform(self, X, y=None, **fit_params):
self.fit(X, y, **fit_params)
return self.transform(X)
def transform(self, X, **transform_params):
'''
Input: A pandas dataframe and a list of column names to extract.
Output: A pandas dataframe containing only the columns of the names passed in.
'''
return X[self.columns]
def fit(self, X, y=None, **fit_params):
return self
def get_params(self, deep=True):
"""Get parameters for this estimator.
Parameters
----------
deep: boolean, optional
If True, will return the parameters for this estimator and
contained subobjects that are estimators.
Returns
-------
params : mapping of string to any
Parameter names mapped to their values.
"""
return self
In [87]:
X_df = df[['total_payments', 'total_stock_value', 'shared_receipt_with_poi', 'bonus']]
y_df = df['poi']
from sklearn.svm import LinearSVC
sk_fold = StratifiedShuffleSplit(y_df, n_iter=10, test_size=0.2)
pipeline = Pipeline(steps=[#('imputer', Imputer(axis=0, copy=True, missing_values='NaN', strategy='median', verbose=0)),
('standardizer', StandardScaler(copy=True, with_mean=True, with_std=True)),
('low_var_remover', VarianceThreshold(threshold=0.1)),
#('feature_selection', LinearSVC()),
('ET', ExtraTreesClassifier(bootstrap=True, compute_importances=None,
criterion='gini', n_estimators=1500, n_jobs=1,
oob_score=True, random_state=None, verbose=0,
max_features='auto', min_samples_split=2,
min_samples_leaf=1))])
params = {'ET__n_estimators': [1500],
'ET__max_features': ['auto', None],
'ET__min_samples_split': [2, 4, 10],
'ET__min_samples_leaf': [1, 2, 5],
#'feature_selection__C': [0.1, 1, 10],
#'ET__criterion' : ['gini', 'entropy'],
#'imputer__strategy': ['median', 'mean'],
'low_var_remover': [0, 0.1, .25, .50, .75]
}
grid_search = GridSearchCV(pipeline, param_grid=params, cv=sk_fold, n_jobs = -1, scoring='f1')
grid_search.fit(X_df, y=y_df)
#test_pred = grid_search.predict(X_test)
#print "Cross_Val_score: ", cross_val_score(grid_search, X_train, y_train)
print "Best Estimator: ", grid_search.best_estimator_
#f1_avg.append(f1_score(y_test, test_pred))
#print "F1: ", f1_score(y_test, test_pred)
#print "Confusion Matrix: "
#print confusion_matrix(y_test, test_pred)
#print "Accuracy Score: ", accuracy_score(y_test, test_pred)
print "Best Params: ", grid_search.best_params_
In [88]:
n_iter = 100
sk_fold = StratifiedShuffleSplit(y_df, n_iter=n_iter, test_size=0.1)
f1_avg = []
for train_index, test_index in sk_fold:
X_train, X_test = X_df.irow(train_index), X_df.irow(test_index)
y_train, y_test = y_df[train_index], y_df[test_index]
grid_search.best_estimator_.fit(X_train, y=y_train)
# pipeline.fit(X_train, y=y_train)
test_pred = grid_search.predict(X_test)
#test_pred = pipeline.predict(X_test)
#print "Cross_Val_score: ", cross_val_score(grid_search, X_train, y_train)
#print "Best Estimator: ", grid_search.best_estimator_
#print f1_score(y_test, test_pred)
f1_avg.append(f1_score(y_test, test_pred))
print sum(f1_avg)/n_iter
In [ ]:
pipeline = Pipeline(steps=[#('imputer', Imputer(axis=0, copy=True, missing_values='NaN', strategy='median', verbose=0)),
#('standardizer', StandardScaler(copy=True, with_mean=True, with_std=True)),
#('low_var_remover', VarianceThreshold(threshold=0.1)),
#('feature_selection', LinearSVC()),
('features', FeatureUnion([
('financial', Pipeline([
('extract', ColumnExtractor(FINANCIAL_FIELDS)),
('scale', StandardScaler()),
('reduce', LinearSVC())
])),
('email', Pipeline([
('extract2', ColumnExtractor(EMAIL_FIELDS)),
('scale2', StandardScaler()),
('reduce2', LinearSVC())
]))
])),
('ET', ExtraTreesClassifier(bootstrap=True, compute_importances=None,
criterion='gini', n_estimators=1500, n_jobs=1,
oob_score=True, random_state=None, verbose=0,
max_features=None, min_samples_split=2,
min_samples_leaf=1))
])
In [ ]:
In [ ]: